# -*- coding: utf-8 -*-

#    Copyright (C) 2015 by Promotux
#                        di Francesco Meloni snc - http://www.promotux.it/

#    Author: Francesco Meloni  <francesco@promotux.it>

#    This file is part of odmCensus.

#    Promogest is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 2 of the License, or
#    (at your option) any later version.

#    Promogest is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.

#    You should have received a copy of the GNU General Public License
#    along with Promogest.  If not, see <http://www.gnu.org/licenses/>.

import urllib2
import urllib
import collections
import csv
import itertools
import base64
import hashlib
import datetime
import Environment
from web_lib.page import Page, expose, Response, change_schema
import json
from dao.Ente import Ente
from dao.Licenza import Licenza
import datetime
from lib.utils import prep2json, dateToString , dateTimeToString
from webEnv import *
from web_lib.webutils import leggiInSessione
import ckanapi

"""
ckan.logic.action.create.organization_create(context, data_dict)

    Create a new organization.

    You must be authorized to create organizations.

    Plugins may change the parameters of this function depending on the value of the type parameter, see the IGroupForm plugin interface.
    Parameters:

        name (string) – the name of the organization, a string between 2 and 100 characters long, containing only lowercase alphanumeric characters, - and _
        id (string) – the id of the organization (optional)
        title (string) – the title of the organization (optional)
        description (string) – the description of the organization (optional)
        image_url (string) – the URL to an image to be displayed on the organization’s page (optional)
        state (string) – the current state of the organization, e.g. 'active' or 'deleted', only active organizations show up in search results and other lists of organizations, this parameter will be ignored if you are not authorized to change the state of the organization (optional, default: 'active')
        approval_status (string) – (optional)
        extras (list of dataset extra dictionaries) – the organization’s extras (optional), extras are arbitrary (key: value) metadata items that can be added to organizations, each extra dictionary should have keys 'key' (a string), 'value' (a string), and optionally 'deleted'
        packages (list of dictionaries) – the datasets (packages) that belong to the organization, a list of dictionaries each with keys 'name' (string, the id or name of the dataset) and optionally 'title' (string, the title of the dataset)
        users (list of dictionaries) – the users that belong to the organization, a list of dictionaries each with key 'name' (string, the id or name of the user) and optionally 'capacity' (string, the capacity in which the user is a member of the organization)

    Returns: the newly created organization
    Return type:   dictionary


ckan.logic.action.create.group_create(context, data_dict)

        Create a new group.

        You must be authorized to create groups.

        Plugins may change the parameters of this function depending on the value of the type parameter, see the IGroupForm plugin interface.
        Parameters:

            name (string) – the name of the group, a string between 2 and 100 characters long, containing only lowercase alphanumeric characters, - and _
            id (string) – the id of the group (optional)
            title (string) – the title of the group (optional)
            description (string) – the description of the group (optional)
            image_url (string) – the URL to an image to be displayed on the group’s page (optional)
            type (string) – the type of the group (optional), IGroupForm plugins associate themselves with different group types and provide custom group handling behaviour for these types Cannot be ‘organization’
            state (string) – the current state of the group, e.g. 'active' or 'deleted', only active groups show up in search results and other lists of groups, this parameter will be ignored if you are not authorized to change the state of the group (optional, default: 'active')
            approval_status (string) – (optional)
            extras (list of dataset extra dictionaries) – the group’s extras (optional), extras are arbitrary (key: value) metadata items that can be added to groups, each extra dictionary should have keys 'key' (a string), 'value' (a string), and optionally 'deleted'
            packages (list of dictionaries) – the datasets (packages) that belong to the group, a list of dictionaries each with keys 'name' (string, the id or name of the dataset) and optionally 'title' (string, the title of the dataset)
            groups (list of dictionaries) – the groups that belong to the group, a list of dictionaries each with key 'name' (string, the id or name of the group) and optionally 'capacity' (string, the capacity in which the group is a member of the group)
            users (list of dictionaries) – the users that belong to the group, a list of dictionaries each with key 'name' (string, the id or name of the user) and optionally 'capacity' (string, the capacity in which the user is a member of the group)

        Returns: the newly created group
        Return type: dictionary




ckan.logic.action.create.package_create(context, data_dict)

    Create a new dataset (package).

    You must be authorized to create new datasets. If you specify any groups for the new dataset, you must also be authorized to edit these groups.

    Plugins may change the parameters of this function depending on the value of the type parameter, see the IDatasetForm plugin interface.
    Parameters:

        name (string) – the name of the new dataset, must be between 2 and 100 characters long and contain only lowercase alphanumeric characters, - and _, e.g. 'warandpeace'
        title (string) – the title of the dataset (optional, default: same as name)
        author (string) – the name of the dataset’s author (optional)
        author_email (string) – the email address of the dataset’s author (optional)
        maintainer (string) – the name of the dataset’s maintainer (optional)
        maintainer_email (string) – the email address of the dataset’s maintainer (optional)
        license_id (license id string) – the id of the dataset’s license, see license_list() for available values (optional)
        notes (string) – a description of the dataset (optional)
        url (string) – a URL for the dataset’s source (optional)
        version (string, no longer than 100 characters) – (optional)
        state (string) – the current state of the dataset, e.g. 'active' or 'deleted', only active datasets show up in search results and other lists of datasets, this parameter will be ignored if you are not authorized to change the state of the dataset (optional, default: 'active')
        type (string) – the type of the dataset (optional), IDatasetForm plugins associate themselves with different dataset types and provide custom dataset handling behaviour for these types
        resources (list of resource dictionaries) – the dataset’s resources, see resource_create() for the format of resource dictionaries (optional)
        tags (list of tag dictionaries) – the dataset’s tags, see tag_create() for the format of tag dictionaries (optional)
        extras (list of dataset extra dictionaries) – the dataset’s extras (optional), extras are arbitrary (key: value) metadata items that can be added to datasets, each extra dictionary should have keys 'key' (a string), 'value' (a string), and optionally 'deleted'
        relationships_as_object (list of relationship dictionaries) – see package_relationship_create() for the format of relationship dictionaries (optional)
        relationships_as_subject (list of relationship dictionaries) – see package_relationship_create() for the format of relationship dictionaries (optional)
        groups (list of dictionaries) – the groups to which the dataset belongs (optional), each group dictionary should have one or more of the following keys which identify an existing group: 'id' (the id of the group, string), 'name' (the name of the group, string), 'title' (the title of the group, string), to see which groups exist call group_list()

    Returns:

    the newly created dataset (unless ‘return_id_only’ is set to True in the context, in which case just the dataset id will be returned)
    Return type:

    dictionary

ckan.logic.action.create.resource_create(context, data_dict)

    Appends a new resource to a datasets list of resources.
    Parameters:

        package_id (string) – id of package that the resource needs should be added to.
        url (string) – url of resource
        revision_id – (optional)
        description (string) – (optional)
        format (string) – (optional)
        hash (string) – (optional)
        name (string) – (optional)
        resource_type (string) – (optional)
        mimetype (string) – (optional)
        mimetype_inner (string) – (optional)
        webstore_url (string) – (optional)
        cache_url (string) – (optional)
        size (int) – (optional)
        created (iso date string) – (optional)
        last_modified (iso date string) – (optional)
        cache_last_updated (iso date string) – (optional)
        webstore_last_updated (iso date string) – (optional)

    Returns:

    the newly created resource
    Return type:


http://docs.ckan.org/en/ckan-2.0/ckan.logic.action.create.html



"""

@expose('/pg/GetLicenzaFromCkan')
def __get_licenza_from_ckan__(req, action=None):
    pageData = {'file': "pg/addedit/ae_opendata_ordina",
                #"daos": daos,
                }
    return Page(req).render(pageData)


def get_licenze_from_ckan():
    #opener = urllib.FancyURLopener({})
    f = urllib.urlopen("http://demo.ckan.org/api/3/action/license_list")
    for k in json.loads(f.read())["result"]:
        lic = Licenza().select(name=k["id"])
        if not lic:
            l = Licenza()
            l.name=k["id"]
            l.descrizione = k["title"]
            l.persist()


def _export_param_(req, action=None):
    """ Questa funzione riceve la selezione delle rilevazioni
    Qui vengono effettuati i primi controlli di coerenza prima dell'export
    """
    #Import a livello di funzione per motivi di multischema
    from dao.ParametriExport import ParametriExport
    from dao.OpenData import OpenData
    from dao.Esportazione import Esportazione
    from dao.Licenza import Licenza

    rilevs = []
    categ = []
    enti = []
    exported = 0

    parex = ParametriExport().select(batchSize=None, orderBy="denominazione")
    #rilevazioni selezionate
    rilev = req.form.get("rilev")
    dd =  eval(json.dumps(rilev)).replace('"',"")

    for d in eval(dd):
        c = OpenData().getRecord(id=int(d))
        rilevs.append(c)
        if c.id_categoria not in categ:
            categ.append(c.id_categoria )
        if c.id_ente not in enti:
            enti.append(c.id_ente)
    print "Elenco delle rilevazioni selezionate", rilev

    for r in rilevs:
        if Esportazione().select(id_opendatastore= r.id):
            exported +=1

    pageData = {'file': "pg/addedit/ae_rilevazioneexportparam",
                "daos": dd,
                "parex": parex,
                "rilev":len(rilevs),
                "enti":len(enti),
                "categ":len(categ),
                "licenze":Licenza().select(batchSize=None),
                "exported":exported,
                }
    print "PAEDATA", pageData
    return Page(req).render(pageData)


def _export_(req, action=None):
    """ Funzione di esporazione vera a propria, qui verranno mandati i dati
    alla piattaforma selezionata

    """
    #import a livello funzione dovuto al multischema
    from dao.OpenData import OpenData
    from dao.CategoriaCampo import CategoriaCampo
    from dao.Campo import Campo
    from dao.ParametriExport import ParametriExport
    from dao.Categoria import Categoria
    from dao.Esportazione import Esportazione
    from dao.Organization import Organization

    rilevs = []
    categ = []
    enti = []
    # Dati relativi al form dove si affinano i parametri di export
    rilev = req.form.get("rilev")
    id_parexp = req.form.get("id_parametri_export")
    autore =  req.form.get("author")
    autore_email =  req.form.get("author_email")
    manutentore = req.form.get("manutentore")
    manutentore_email = req.form.get("manutentore_email")
    url = req.form.get("url")
    licenza = req.form.get("id_licenza")
    print " LICENZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", licenza
    new_package = req.form.get("new_package")
    CSV = req.form.get("CSV")
    XML = req.form.get("XML")
    JSON = req.form.get("JSON")
    force_name_dataset = req.form.get("force_name_dataset")
    discard = req.form.get("discard")


    parexp = ParametriExport().getRecord(id=int(id_parexp))
    orgs = Organization().getRecord(id=parexp.id_organization)
    # COstruisco il CSV
    dd =  eval(json.dumps(rilev)).replace('"',"")
    for d in eval(dd):
        c = OpenData().getRecord(id=int(d))
        if bool(discard) == True: #  and not Esportazione().select(id_opendatastore= c.id):
            rilevs.append(c)
        else:
            if c and not Esportazione().select(id_opendatastore= c.id):
                rilevs.append(c)
        if c.id_categoria not in categ:
            categ.append(c.id_categoria )
        if c.id_ente not in enti:
            enti.append(c.id_ente)
    tai = CategoriaCampo().select(id_categoria=int(categ[0]),orderBy="ordine", batchSize=None)
    item = [str(d.id_campo) for d in tai]
    campi = []
    for i in item:
        aa = Campo().getRecord(id=i)
        campi.append(aa)
    categoria = Categoria().getRecord(id=int(categ[0]))
    ready_to_export = []
    for r in rilevs:
        jj = json.dumps(r.objson)
        singola_rilevazione_dict =  eval(eval(jj))[0]
        prepped = collections.OrderedDict()

        for d in campi:
            if d.denominazione in singola_rilevazione_dict:
                prepped[d.denominazione] = singola_rilevazione_dict[d.denominazione]
        ready_to_export.append(prepped)
    fieldnames = [d.denominazione for d in campi]
    test_file = open('test2.csv','wb')
    csvwriter = csv.DictWriter(test_file, delimiter=',', fieldnames=fieldnames)
    csvwriter.writerow(dict((fn,fn) for fn in fieldnames))
    for row in ready_to_export:
         csvwriter.writerow(row)
    test_file.close()

    # CHIUDO IL CSV ...mi preparo a spedire
    # CREO JSON DA CSV

    csvfile = open('test2.csv', 'r')
    jsonfile = open('test2.json', 'wb')


    reader = csv.DictReader( csvfile, fieldnames)
    rows = []
    for row in reader:
        rows.append(row)
    json.dump(rows, jsonfile,sort_keys=True, indent=4, separators=(',', ': '), ensure_ascii=False, encoding='utf8')
    jsonfile.write('\n')
    jsonfile.close()
    #FINE CREAZIONE JSON

    #CREO XML

    csvFile = 'test2.csv'
    xmlFile = 'test2.xml'

    csvData = csv.reader(open(csvFile))
    xmlData = open(xmlFile, 'wb')
    xmlData.write('<?xml version="1.0"?>' + "\n")
    # there must be only one top-level tag
    xmlData.write('<csv_data>' + "\n")

    rowNum = 0
    for row in csvData:
        if rowNum == 0:
            tags = row
            # replace spaces w/ underscores in tag names
            for i in range(len(tags)):
                tags[i] = tags[i].replace(' ', '_')
        else:
            xmlData.write('<row>' + "\n")
            for i in range(len(tags)):
                xmlData.write('    ' + '<' + tags[i] + '>' \
                              + row[i] + '</' + tags[i] + '>' + "\n")
            xmlData.write('</row>' + "\n")

        rowNum +=1

    xmlData.write('</csv_data>' + "\n")
    xmlData.close()

    # FINE CREAZIONE XML


    # Istanzio l'oggetto ckanapi per poterci lavorare ...sarà da spostare?
    mysite = ckanapi.RemoteCKAN(parexp.url_destinazione,
        apikey=parexp.apikey,
        user_agent='brainODM/collect 1.0 (+http://opendatagpf.it)')

    #Aggiungiamo una organizzazione prendendola dalla anagrafica apposita
    try:
        org = mysite.action.organization_create(name=orgs.name,
                                    title=orgs.title,
                                    description=orgs.description,
                                    state=orgs.state,
                                    image_url=orgs.image_url,
                                    )
    except:
        print "ORGANIZZAZIONE GIA' PRESENTE"

    #aggiungiamo un gruppo prendendolo dai cataloghi o tipi categoria
    try:
        print "GRUPPO", categoria.tipo_categoria.lower()
        mysite.action.group_create(name=categoria.tipo_categoria.replace(" ", "_").strip().lower(),
                                title=categoria.tipo_categoria,
                                description=categoria.tipo_cate.descrizione,
                                state="active",
                                image_url=categoria.tipo_cate.image_url,
                                    )
    except:
        print "GRuppo GIa PRESENTE"


    #Proviamo a creare un package.
    if new_package:
        try:
            pkg = mysite.action.package_create( owner_org=orgs.name,
                                                notes=categoria.descrizione,
                                                author = autore,
                                                author_email =autore_email,
                                                maintainer = manutentore,
                                                maintainer_email = manutentore_email,
                                                title = categoria.denominazione,
                                                license_id =licenza,
                                                groups = [{"name":categoria.tipo_categoria.replace(" ", "_").strip().lower()}],
                                                url = url)
            if CSV:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="CSV",
                        upload=open('test2.csv'))
            if JSON:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="JSON",
                        upload=open('test2.json'))
            if XML:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="XML",
                        upload=open('test2.xml'))
        except:
            if force_name_dataset != "":
                name = force_name_dataset.replace(" ", "_").lower().replace("/", "_").replace(":"," ")
            else:
                name = categoria.denominazione.replace(" ", "_").lower()+"_"+dateTimeToString(datetime.datetime.now()).replace("/", "_").replace(" ","").replace(":","")
                force_name_dataset =  name
            pkg = mysite.action.package_create( owner_org=orgs.name,
                                                name=name,
                                                notes=categoria.descrizione,
                                                author = autore,
                                                author_email = autore_email,
                                                maintainer = manutentore,
                                                maintainer_email = manutentore_email,
                                                title = force_name_dataset,
                                                license_id = licenza,
                                                groups = [{"name":categoria.tipo_categoria.replace(" ", "_").strip().lower()}],
                                                url = url)
            if CSV:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="CSV",
                        upload=open('test2.csv'))
            if JSON:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="JSON",
                        upload=open('test2.json'))
            if XML:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="XML",
                        upload=open('test2.xml'))
    else:
        try:
            pkg = mysite.action.package_create( owner_org=orgs.name,
                                                name = categoria.denominazione.replace(" ", "_").lower()+"_"+dateTimeToString(datetime.datetime.now()).replace("/", "_").replace(" ","").replace(":",""),
                                                notes=categoria.descrizione,
                                                author = autore,
                                                author_email = autore_email,
                                                maintainer = manutentore,
                                                maintainer_email = manutentore_email,
                                                license_id =licenza,
                                                title = categoria.denominazione.replace(" ", "_").lower(),
                                                groups = [{"name":categoria.tipo_categoria.replace(" ", "_").strip().lower()}],
                                                url = url)
            if CSV:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="CSV",
                        upload=open('test2.csv'))
            if JSON:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="JSON",
                        upload=open('test2.json'))
            if XML:
                mysite.action.resource_create(
                        package_id=pkg["id"],
                        name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "-"),
                        format="XML",
                        upload=open('test2.xml'))
        except:
            hdr = {'User-Agent' : "Magic Browser"}
            try:
                request = urllib2.Request(
                            parexp.url_destinazione+"/api/3/action/package_show?id="+categoria.denominazione.replace(" ", "_").lower() ,headers=hdr)
                response = urllib2.urlopen(request)
                response_dict = json.loads(response.read())
                if CSV:
                    mysite.action.resource_create(
                            package_id=response_dict["result"]["id"],
                            name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "_"),
                            format="CSV",
                            groups = [{"name":categoria.tipo_categoria.replace(" ", "_").strip().lower()}],
                            upload=open('test2.csv'))
                if JSON:
                    mysite.action.resource_create(
                            package_id=response_dict["result"]["id"],
                            name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "_"),
                            format="JSON",
                            groups = [{"name":categoria.tipo_categoria.replace(" ", "_").strip().lower()}],
                            upload=open('test2.json'))
                if XML:
                    mysite.action.resource_create(
                            package_id=response_dict["result"]["id"],
                            name=str(categoria.denominazione)+"_"+dateToString(datetime.datetime.now()).replace("/", "_"),
                            format="XML",
                            groups = [{"name":categoria.tipo_categoria.replace(" ", "_").strip().lower()}],
                            upload=open('test2.xml'))
            except:
                url_red = "/pg/ac_Rilevazioni"
                return Page(req).redirect(url_red)


    #salvo l'esportazione
    for g in rilevs:
        print g
        d = Esportazione()
        d.id_opendatastore = g.id
        d.id_parametri_export = id_parexp
        d.data_esportazione = datetime.datetime.now()
        Environment.session.add(d)
    Environment.session.commit()
    print " SALVO ESPORTAZIONE"
    url_red = "/pg/ac_Rilevazioni"
    return Page(req).redirect(url_red)
